import fileinput
import MySQLdb
import sys

DOMAIN_NAME = 0
CLIENT_IP = 1                  
COMPLETION_TIME = 3    
HTTP_REQUEST = 4       
STATUS_CODE = 5
RETURNED_SIZE = 6
REFERRER = 7
USER_AGENT = 8
#count = 0
db = MySQLdb.connect("localhost", "root", "", "irec")
cursor = db.cursor()

def scanlogfile():
    #infile = open('D:\\format.txt')
    #for linenumber, line in enumerate(infile.read().split("\n")):
    count =0
    lis=[]
      
    for line in fileinput.input(['D:\\access_log.25']):
       try:
           
           if len(line) == 0:
               continue
           row = getlogfields(line)
     
           expandedrow = expandedfields(row)
           
           if len(row) == 9 and len(expandedrow) >= 2:
               count = count + 1
               print "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC             ", count
               if count == 2000:
                  
                   print "XXXXXXXXXXXx"
                   le = len(lis)
                   print lis[le-1]
                   break
                   
               li = [row[0],expandedrow[0],row[2],expandedrow[1],row[4],row[5],row[6] , row[7], row[8]]
               lis.append(li)
               
#               sql = "INSERT INTO logdata(Hostname, IP, UserId,Time, Request, Response, Downloadvol, Referrer, Useragent) VALUES ('" + row[0] + "','" + expandedrow[0] + "','" + row[2] + "','" + expandedrow[1] + "','" + row[4] + "','" + row[5] + "','" + row[6] + "','" + row[7] + "','" + row[8] + "')"
#               sql = "INSERT INTO logdata(Hostname, IP, UserId,Time, Request, Response, Downloadvol, Referrer, Useragent) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s','%s')"
       except:
           continue    
#       try:
##           cursor.execute(sql, row[0], expandedrow[0], row[2], expandedrow[1], row[4], row[5], row[6], row[7], row[8])
##           cursor.execute(sql)
##           db.commit()
#       except:
#           db.rollback()
#           continue
       #elif len(row) != 0: # an empty line, especially a trailing line,  is not an error!
           #nerrorlines += 1
           #print "*******ERROR:**********", len(row)
           #print line, row
    #return (rows, linenumber, nerrorlines)
    sql = "INSERT INTO logdata1(Hostname, IP, UserId,Time, Request, Response, Downloadvol, Referrer, Useragent) VALUES "
    for list in lis:
        sql=sql+"('"+list[0]+"','"+list[1]+"','"+list[2]+"','"+list[3]+"','"+list[4]+"','"+list[5]+"','"+list[6]+"','"+list[7]+"','"+list[8]+"'),"
    sq = sql[:-1]
    print sq
    try:
#           cursor.execute(sql, row[0], expandedrow[0], row[2], expandedrow[1], row[4], row[5], row[6], row[7], row[8])
           cursor.execute(sq)
           db.commit()
    except:
           db.rollback()
    db.close()

def getlogfields(s):
    fields = []
    # error! no leading spaces allowed!
    if s.startswith(" "): 
       return fields

    state = 0
    quoteds = ""
    n = 0
    for f in s.split():
        if state == 0:
           if f.startswith("\""):
               if f.endswith("\""):
                   fields.append(f[1:-1])
               else:
                   quoteds = f[1:] + " "
                   state = 2
               continue
           elif f.startswith("'"):
               if f.endswith("\""):
                   fields.append(f[1:-1])
               else:
                   quoteds = f[1:] + " "
                   state = 3
               continue
           elif f.startswith("["):
               if f.endswith("]"):
                   fields.append(f[1:-1])
               else:
                   blockeds = f[1:] + " "
                   state = 4
               continue
           else:
               fields.append(f)
 
        if state == 2 :
            quoteds += (f + " ")
            if f.endswith("\"") :
               quoteds = quoteds[:-2]
               fields.append(quoteds)
               n += 1
               state = 0
 
        if state == 3 :
            quoteds += (f + " ")
            if f.endswith("'") :
               quoteds = quoteds[:-2]
               fields.append(quoteds)
               n += 1
               state = 0
 
        if state == 4 :
            blockeds += (f + " ") 
            if f.endswith("]") :
               blockeds = blockeds[:-2]
               fields.append(blockeds)
               n += 1
               state = 0
 
        #@@@ print "processing [", f, "]; state=", state,";current=",  fields, "quoteds=", quoteds
    return fields

def logsplittimefield(timefield):
    try:
        print "debugging...", timefield
        (stime, zone) = timefield.split()
        caldate, hour, minute, second = stime[0:].split(":")
        day, month, year = caldate.split("/")
    except:
        print "error procedding timefield [%s]\n" % timefield
        return None, None, None, None, None, None, None
    return (day, month, year, hour, minute, second, zone)

def logsplitrequestfield(requestfield):
    try:
        # you can avoid the intermediate storing to variables.
        method, resource, protocol = requestfield.split()
    except:
        return None, None, None
 
    return method, resource, protocol
 
 
def logsplituseragentfield(useragentfield):
    """
    Returns (browser, url)
    """
    return useragentfield.split()

def expandedfields(ninefields):
    """
    Returns a full tuple of information.
    """
    try:
        if len(ninefields) == 0:
            return None, None
        ip1, ip2 = ninefields[CLIENT_IP].split(",")
        if ip1 == '-':
            ip = ip2
        else:
            ip = ip1
        day, month, year, hour, minute, second, zone = logsplittimefield(ninefields[COMPLETION_TIME])
        timestamp = year + "-" + getmonth(month) + "-" + day + " " + hour + ":" + minute + ":" + second
        method, resource, protocol = logsplitrequestfield(ninefields[HTTP_REQUEST])
        status = ninefields[STATUS_CODE]
        size = ninefields[RETURNED_SIZE]
        for i, f in enumerate(ninefields):
            print i, f
        browserandurls = ninefields[USER_AGENT].split()
        browser = browserandurls[0]
        if len(browserandurls) == 1:
            urls = []
        else:
            urls = browserandurls[1:]
            urls[0] = urls[0][1:]
            urls[-1] = urls[-1][:-1]

        return (ip, timestamp, day, month, year, hour, minute, second, zone, method, resource,
                protocol, browser, urls)
    except:
        print "error in processing [%s]" % ninefields
        return None, None, None

def getmonth(month):
    if month == "Jan":
        return "01"
    elif month == "Feb":
        return "02"
    elif month == "Mar":
        return "03"
    elif month == "Apr":
        return "04"
    elif month == "May":
        return "05"
    elif month == "Jun":
        return "06"
    elif month == "Jul":
        return "07"
    elif month == "Aug":
        return "08"
    elif month == "Sep":
        return "09"
    elif month == "Oct":
        return "10"
    elif month == "Nov":
        return "11"
    elif month == "Dec":
        return "12"
    else:
        return "00"
    
if __name__ == "__main__": scanlogfile()
